/*
 * Copyright (C) by Argonne National Laboratory
 *     See COPYRIGHT in top-level directory
 */

/**
 * \file ad_bg_aggrs.c
 * \brief The externally used function from this file is is declared in ad_bg_aggrs.h
 */

/*#define TRACE_ON */

// Uncomment this line to turn tracing on for the gpfsmpio_balancecontig aggr selection optimization
// #define balancecontigtrace 1
// #define bridgeringaggtrace 1

#include "adio.h"
#include "adio_cb_config_list.h"
#include "../ad_gpfs.h"
#include "ad_bg_pset.h"
#include "ad_bg_aggrs.h"
#ifdef AGGREGATION_PROFILE
#include "mpe.h"
#endif


#ifdef MPL_USE_DBG_LOGGING
#define AGG_DEBUG 1
#endif

#ifndef TRACE_ERR
#define TRACE_ERR(format...)
#endif

/* Comments copied from common:
 * This file contains four functions:
 *
 * ADIOI_Calc_aggregator()
 * ADIOI_Calc_file_domains()
 * ADIOI_Calc_my_req()
 * ADIOI_Calc_others_req()
 *
 * The last three of these were originally in ad_read_coll.c, but they are
 * also shared with ad_write_coll.c.  I felt that they were better kept with
 * the rest of the shared aggregation code.
 */

/* Discussion of values available from above:
 *
 * ADIO_Offset st_offsets[0..nprocs-1]
 * ADIO_Offset end_offsets[0..nprocs-1]
 *    These contain a list of start and end offsets for each process in
 *    the communicator.  For example, an access at loc 10, size 10 would
 *    have a start offset of 10 and end offset of 19.
 * int nprocs
 *    number of processors in the collective I/O communicator
 * ADIO_Offset min_st_offset
 * ADIO_Offset fd_start[0..nprocs_for_coll-1]
 *    starting location of "file domain"; region that a given process will
 *    perform aggregation for (i.e. actually do I/O)
 * ADIO_Offset fd_end[0..nprocs_for_coll-1]
 *    start + size - 1 roughly, but it can be less, or 0, in the case of
 *    uneven distributions
 */

/* forward declaration */
static void
ADIOI_BG_compute_agg_ranklist_serial(ADIO_File fd,
                                     const ADIOI_BG_ConfInfo_t * confInfo,
                                     ADIOI_BG_ProcInfo_t * all_procInfo);

/*
 * Compute the aggregator-related parameters that are required in 2-phase collective IO of ADIO.
 * The parameters are
 *  . the number of aggregators (proxies) : fd->hints->cb_nodes
 *  . the ranks of the aggregators :        fd->hints->ranklist
 * By compute these two parameters in a BG-PSET-aware way, the default 2-phase collective IO of
 *  ADIO can work more efficiently.
 */
int ADIOI_BG_gen_agg_ranklist(ADIO_File fd, int n_aggrs_per_pset)
{
    int r, s;
    ADIOI_BG_ProcInfo_t *procInfo, *all_procInfo;
    ADIOI_BG_ConfInfo_t *confInfo;
    TRACE_ERR("Entering ADIOI_BG_gen_agg_ranklist\n");

    MPI_Comm_size(fd->comm, &s);
    MPI_Comm_rank(fd->comm, &r);

    /* Collect individual BG personality information */
    confInfo = ADIOI_BG_ConfInfo_new();
    procInfo = ADIOI_BG_ProcInfo_new();
    ADIOI_BG_persInfo_init(confInfo, procInfo, s, r, n_aggrs_per_pset, fd->comm);

    /* Gather BG personality information onto process 0 */
    /* if (r == 0) */
    all_procInfo = ADIOI_BG_ProcInfo_new_n(s);

    MPI_Gather((void *) procInfo, sizeof(ADIOI_BG_ProcInfo_t), MPI_BYTE,
               (void *) all_procInfo, sizeof(ADIOI_BG_ProcInfo_t), MPI_BYTE, 0, fd->comm);

    /* Compute a list of the ranks of chosen IO proxy CN on process 0 */
    if (r == 0) {
        ADIOI_BG_compute_agg_ranklist_serial(fd, confInfo, all_procInfo);
        /* ADIOI_BG_ProcInfo_free (all_procInfo); */
    }
    ADIOI_BG_ProcInfo_free(all_procInfo);

    /* Send the info of IO proxy CN to all processes and keep the info in fd->hints struct.
     * Declared in adio_cb_config_list.h */
    ADIOI_cb_bcast_rank_map(fd);
    if (gpfsmpio_balancecontig == 1) {  /* additionally need to send bridgelist,
                                         * bridgelistnum and numbridges to all
                                         * ranks */
        if (r != 0) {
            fd->hints->fs_hints.bg.bridgelist = ADIOI_Malloc(fd->hints->cb_nodes * sizeof(int));
            if (fd->hints->fs_hints.bg.bridgelist == NULL) {
                /* NEED TO HANDLE ENOMEM */
            }
        }
        MPI_Bcast(fd->hints->fs_hints.bg.bridgelist, fd->hints->cb_nodes, MPI_INT, 0, fd->comm);

        if (r != 0) {
            fd->hints->fs_hints.bg.bridgelistnum = ADIOI_Malloc(fd->hints->cb_nodes * sizeof(int));
            if (fd->hints->fs_hints.bg.bridgelistnum == NULL) {
                /* NEED TO HANDLE ENOMEM */
            }
        }
        MPI_Bcast(fd->hints->fs_hints.bg.bridgelistnum, fd->hints->cb_nodes, MPI_INT, 0, fd->comm);

        MPI_Bcast(&fd->hints->fs_hints.bg.numbridges, 1, MPI_INT, 0, fd->comm);

    }


    ADIOI_BG_persInfo_free(confInfo, procInfo);
    TRACE_ERR("Leaving ADIOI_BG_gen_agg_ranklist\n");
    return 0;
}


/* There are some number of bridge nodes (randomly) distributed through the job
 * We need to split the nodes among the bridge nodes */
/* Maybe find which bridge node is closer (manhattan distance) and try to
 * distribute evenly.
 */
/*
 * Pick IO aggregators based on the under PSET organization and stores the ranks of the proxy CNs in tmp_ranklist.
 * The first order of tmp_ranklist is : PSET number
 * The secondary order of the list is determined in ADIOI_BG_select_agg_in_pset() and thus adjustable.
 */
typedef struct {
    int rank;
    int bridge;
} sortstruct;

typedef struct {
    int bridgeRank;
    int numAggsAssigned;
} bridgeAggAssignment;

static int intsort(const void *p1, const void *p2)
{
    sortstruct *i1, *i2;
    i1 = (sortstruct *) p1;
    i2 = (sortstruct *) p2;
    return (i1->bridge - i2->bridge);
}

static int
ADIOI_BG_compute_agg_ranklist_serial_do(const ADIOI_BG_ConfInfo_t * confInfo,
                                        ADIOI_BG_ProcInfo_t * all_procInfo, int *tmp_ranklist)
{
    TRACE_ERR("Entering ADIOI_BG_compute_agg_ranklist_serial_do\n");
    /* BES: This should be done in the init routines probably. */
    int i, j;
    int aggTotal;
    int *aggList;

    if (gpfsmpio_bridgeringagg > 0) {

        int numAggs = confInfo->aggRatio * confInfo->ioMinSize /*virtualPsetSize */ ;
        /* the number of aggregators is (numAggs per bridgenode) */
        if (numAggs == 1)
            aggTotal = 1;
        else
            aggTotal = confInfo->numBridgeRanks * numAggs;

        aggList = (int *) ADIOI_Malloc(aggTotal * sizeof(int));
        if (aggTotal == 1) {    /* special case when we only have one bridge node */

            sortstruct *bridgelist =
                (sortstruct *) ADIOI_Malloc(confInfo->nProcs * sizeof(sortstruct));
            for (i = 0; i < confInfo->nProcs; i++) {
                bridgelist[i].bridge = all_procInfo[i].bridgeRank;
                bridgelist[i].rank = i;
                TRACE_ERR("bridgelist[%d].bridge: %d .rank: %d\n", i, bridgelist[i].bridge, i);
            }

            /* This list contains rank->bridge info. Now, we need to sort this list. */
            qsort(bridgelist, confInfo->nProcs, sizeof(sortstruct), intsort);

            aggList[0] = bridgelist[0].bridge;
            ADIOI_Free(bridgelist);

        } else {        // aggTotal > 1

            int currentAggListSize = 0;
            int numBridgesWithAggAssignments = 0;
            bridgeAggAssignment *aggAssignments =
                (bridgeAggAssignment *) ADIOI_Malloc(confInfo->numBridgeRanks *
                                                     sizeof(bridgeAggAssignment));

            int partitionSize = all_procInfo[0].numNodesInPartition;
            int *nodesAssigned = (int *) ADIOI_Malloc(partitionSize * sizeof(int));
            for (i = 0; i < partitionSize; i++)
                nodesAssigned[i] = 0;

            int currentNumHops = gpfsmpio_bridgeringagg;
            int allAggsAssigned = 0;

            /* Iterate thru the process infos and select aggregators starting at currentNumHops
             * away.  Increase the currentNumHops until all bridges have numAggs assigned to them.
             */
            while (!allAggsAssigned) {
                /* track whether any aggs are selected durng this round */
                int startingCurrentAggListSize = currentAggListSize;
                int numIterForHopsWithNoAggs = 0;
                for (i = 0; i < confInfo->nProcs; i++) {
                    if (all_procInfo[i].manhattanDistanceToBridge == currentNumHops) {
                        if (nodesAssigned[all_procInfo[i].nodeRank] == 0) {     // node is not assigned as an agg yet
                            int foundBridge = 0;
                            for (j = 0; (j < numBridgesWithAggAssignments && !foundBridge); j++) {
                                if (aggAssignments[j].bridgeRank == all_procInfo[i].bridgeRank) {
                                    foundBridge = 1;
                                    if (aggAssignments[j].numAggsAssigned < numAggs) {
                                        aggAssignments[j].numAggsAssigned++;
                                        nodesAssigned[all_procInfo[i].nodeRank] = 1;
                                        aggList[currentAggListSize] = all_procInfo[i].rank;
                                        currentAggListSize++;
#ifdef bridgeringaggtrace
                                        printf
                                            ("Assigned agg rank %d at nodeRank %d to bridge rank %d at a distance of %d hops\n",
                                             all_procInfo[i].rank, all_procInfo[i].nodeRank,
                                             all_procInfo[i].bridgeRank, currentNumHops);
#endif
                                    }
                                }
                            }
                            if (!foundBridge) {
                                aggAssignments[numBridgesWithAggAssignments].bridgeRank =
                                    all_procInfo[i].bridgeRank;
                                aggAssignments[numBridgesWithAggAssignments].numAggsAssigned = 1;
                                numBridgesWithAggAssignments++;
                                nodesAssigned[all_procInfo[i].nodeRank] = 1;
                                aggList[currentAggListSize] = all_procInfo[i].rank;
                                currentAggListSize++;
#ifdef bridgeringaggtrace
                                printf
                                    ("Assigned agg rank %d at nodeRank %d to bridge rank %d at a distance of %d hops\n",
                                     all_procInfo[i].rank, all_procInfo[i].nodeRank,
                                     all_procInfo[i].bridgeRank, currentNumHops);
#endif
                            }
                        }
                    }
                }

                if (numBridgesWithAggAssignments == confInfo->numBridgeRanks) {
                    allAggsAssigned = 1;
                    for (i = 0; (i < numBridgesWithAggAssignments && allAggsAssigned); i++) {
                        if (aggAssignments[i].numAggsAssigned < numAggs)
                            allAggsAssigned = 0;
                    }
                }
                currentNumHops++;
                /* Handle the case where the numAggs is more than exists starting
                 * at gpfsmpio_bridgeringagg hops, wrap back and restart at 0 to
                 * assign the overrun - it is up to the user to realize this
                 * situation and adjust numAggs and gpfsmpio_bridgeringagg
                 * accordingly.
                 */
                if (currentNumHops > 16)
                    currentNumHops = 0;
                /* If 3 rounds go by without selecting an agg abort to avoid
                 * infinite loop.
                 */
                if (startingCurrentAggListSize == currentAggListSize)
                    numIterForHopsWithNoAggs++;
                else
                    numIterForHopsWithNoAggs = 0;
                ADIOI_Assert(numIterForHopsWithNoAggs <= 3);
            }

            ADIOI_Free(aggAssignments);
            ADIOI_Free(nodesAssigned);

        }       // else aggTotal  > 1

        memcpy(tmp_ranklist, aggList, aggTotal * sizeof(int));
    }   // gpfsmpio_bridgeringagg > 0

    else {      // gpfsmpio_bridgeringagg unset - default code

        int distance, numAggs;

        /* Aggregators will be midpoints between sorted MPI rank lists of who shares a given
         * bridge node */

        sortstruct *bridgelist = (sortstruct *) ADIOI_Malloc(confInfo->nProcs * sizeof(sortstruct));
        for (i = 0; i < confInfo->nProcs; i++) {
            bridgelist[i].bridge = all_procInfo[i].bridgeRank;
            bridgelist[i].rank = i;
            TRACE_ERR("bridgelist[%d].bridge: %d .rank: %d\n", i, bridgelist[i].bridge, i);
        }

        /* This list contains rank->bridge info. Now, we need to sort this list. */
        qsort(bridgelist, confInfo->nProcs, sizeof(sortstruct), intsort);

        /* In this array, we can pick an appropriate number of midpoints based on
         * our bridgenode index and the number of aggregators */

        numAggs = confInfo->aggRatio * confInfo->ioMinSize /*virtualPsetSize */ ;
        if (numAggs == 1)
            aggTotal = 1;
        else
            /* the number of aggregators is (numAggs per bridgenode) plus each
             * bridge node is an aggregator */
            aggTotal = confInfo->numBridgeRanks * (numAggs + 1);

        if (aggTotal > confInfo->nProcs)
            aggTotal = confInfo->nProcs;

        TRACE_ERR
            ("numBridgeRanks: %d, aggRatio: %f numBridge: %d pset size: %d/%d numAggs: %d, aggTotal: %d\n",
             confInfo->numBridgeRanks, confInfo->aggRatio, confInfo->numBridgeRanks,
             confInfo->ioMinSize, confInfo->ioMaxSize /*virtualPsetSize */ , numAggs, aggTotal);
        aggList = (int *) ADIOI_Malloc(aggTotal * sizeof(int));


        /* For each bridge node, determine who the aggregators will be */
        /* basically, the n*distance and bridge node */
        if (aggTotal == 1)      /* special case when we only have one bridge node */
            aggList[0] = bridgelist[0].bridge;
        else {
            int lastBridge = bridgelist[confInfo->nProcs - 1].bridge;
            int nextBridge = 0, nextAggr = confInfo->numBridgeRanks;
            int psetSize = 0;
            int procIndex;
            for (procIndex = confInfo->nProcs - 1; procIndex >= 0; procIndex--) {
                TRACE_ERR("bridgelist[%d].bridge %u/rank %u\n", procIndex,
                          bridgelist[procIndex].bridge, bridgelist[procIndex].rank);
                if (lastBridge == bridgelist[procIndex].bridge) {
                    psetSize++;
                    if (procIndex)
                        continue;
                    else
                        procIndex--;    /* procIndex == 0 */
                }
                /* Sets up a list of nodes which will act as aggregators. numAggs
                 * per bridge node total. The list of aggregators is
                 * bridgeNode 0
                 * bridgeNode 1
                 * bridgeNode ...
                 * bridgeNode N
                 * bridgeNode[0]aggr[0]
                 * bridgeNode[0]aggr[1]...
                 * bridgeNode[0]aggr[N]...
                 * ...
                 * bridgeNode[N]aggr[0]..
                 * bridgeNode[N]aggr[N]
                 */
                aggList[nextBridge] = lastBridge;
                distance = psetSize / numAggs;
                TRACE_ERR("nextBridge %u is bridge %u, distance %u, size %u\n", nextBridge,
                          aggList[nextBridge], distance, psetSize);
                if (numAggs > 1) {
                    for (j = 0; j < numAggs; j++) {
                        ADIOI_Assert(nextAggr < aggTotal);
                        aggList[nextAggr] = bridgelist[procIndex + j * distance + 1].rank;
                        TRACE_ERR("agglist[%d] -> bridgelist[%d] = %d\n", nextAggr,
                                  procIndex + j * distance + 1, aggList[nextAggr]);
                        if (aggList[nextAggr] == lastBridge) {  /* can't have bridge in the list twice */
                            aggList[nextAggr] = bridgelist[procIndex + psetSize].rank;  /* take the last one in the pset */
                            TRACE_ERR("replacement agglist[%d] -> bridgelist[%d] = %d\n", nextAggr,
                                      procIndex + psetSize, aggList[nextAggr]);
                        }
                        nextAggr++;
                    }
                }
                if (procIndex < 0)
                    break;
                lastBridge = bridgelist[procIndex].bridge;
                psetSize = 1;
                nextBridge++;
            }
        }

        TRACE_ERR
            ("memcpy(tmp_ranklist, aggList, (numAggs(%u)*confInfo->numBridgeRanks(%u)+numAggs(%u)) (%u) %u*sizeof(int))\n",
             numAggs, confInfo->numBridgeRanks, numAggs,
             (numAggs * confInfo->numBridgeRanks + numAggs), aggTotal);
        memcpy(tmp_ranklist, aggList, aggTotal * sizeof(int));
        for (i = 0; i < aggTotal; i++) {
            TRACE_ERR("tmp_ranklist[%d]: %d\n", i, tmp_ranklist[i]);
        }


        ADIOI_Free(bridgelist);

        TRACE_ERR("Leaving ADIOI_BG_compute_agg_ranklist_serial_do\n");
    }

    ADIOI_Free(aggList);
    return aggTotal;

}

/*
 * compute aggregators ranklist and put it into fd->hints struct
 */
static void
ADIOI_BG_compute_agg_ranklist_serial(ADIO_File fd,
                                     const ADIOI_BG_ConfInfo_t * confInfo,
                                     ADIOI_BG_ProcInfo_t * all_procInfo)
{
    TRACE_ERR("Entering ADIOI_BG_compute_agg_ranklist_serial\n");
    int i;
    int naggs;
    int size;
    int *tmp_ranklist;

    /* compute the ranklist of IO aggregators and put into tmp_ranklist */
    tmp_ranklist = (int *) ADIOI_Malloc(confInfo->nProcs * sizeof(int));

#if AGG_DEBUG
    for (i = 0; i < confInfo->nProcs; i++) {
        DBG_FPRINTF(stderr, "\trank = %6d\n", all_procInfo[i].rank);
    }
#endif

    naggs = ADIOI_BG_compute_agg_ranklist_serial_do(confInfo, all_procInfo, tmp_ranklist);

#define VERIFY 1
#if VERIFY
    DBG_FPRINTF(stderr,
                "\tconfInfo = min: %3d, max: %3d, naggrs: %3d, bridge: %3d, nprocs: %3d, vpset: %3d, ratio: %.4f; naggs = %d\n",
                confInfo->ioMinSize, confInfo->ioMaxSize, confInfo->nAggrs,
                confInfo->numBridgeRanks, confInfo->nProcs,
                confInfo->ioMaxSize /*virtualPsetSize */ ,
                confInfo->aggRatio, naggs);
#endif
    MPI_Comm_size(fd->comm, &size);
    /* This fix is for when the bridgenode rnk is not part of the particular
     * subcomm associated with this MPI File operation. I don't know if
     * this is the best/right answer but it passes the test cases at least.
     * I don't know how common file IO in subcomms is anyway... */
    for (i = 0; i < naggs; i++) {
        if (tmp_ranklist[i] > size) {
            TRACE_ERR("Using 0 as tmp_ranklist[%d] instead of %d for comm %x\n",
                      i, tmp_ranklist[i], fd->comm);
            tmp_ranklist[i] = 0;
        }
    }

#if AGG_DEBUG
    for (i = 0; i < naggs; i++) {
        DBG_FPRINTF(stderr, "\taggr %-4d = %6d\n", i, tmp_ranklist[i]);
    }
#endif
    if (gpfsmpio_balancecontig == 1) {
        /* what comes out of this code block is the agg ranklist sorted by
         * bridge set and ion id with associated bridge info stored in the
         * hints structure for later access during file domain assignment */

        // sort the agg ranklist by ions and bridges

        int *interleavedbridgeranklist = (int *) ADIOI_Malloc(naggs * sizeof(int));     // resorted agg rank list
        /* list of all bridge ranks */
        int *bridgelist = (int *) ADIOI_Malloc(naggs * sizeof(int));

        /* each entry here is the number of aggregators associated with the
         * bridge rank of the same index in bridgelist */
        int *bridgelistnum = (int *) ADIOI_Malloc(naggs * sizeof(int));
        /* list of all ion IDs corresponding with bridgelist entries of same index */
        int *ionlist = (int *) ADIOI_Malloc(naggs * sizeof(int));

        int numbridges = 0;

        for (i = 0; i < naggs; i++)
            bridgelistnum[i] = 0;

        /* Each entry in this list corresponds with the bridgelist and will contain the lowest bridge
         * agg rank on that ion. */
        int *summarybridgeminionaggrank = (int *) ADIOI_Malloc(naggs * sizeof(int));
        for (i = 0; i < naggs; i++)
            summarybridgeminionaggrank[i] = -1;

        /* build the bridgelist, ionlist and bridgelistnum data by going thru each agg
         * entry and find the associated bridge list index - at the end we will
         * know how many aggs belong to each bridge in each ion */
        for (i = 0; i < naggs; i++) {
            int aggbridgerank = all_procInfo[tmp_ranklist[i]].bridgeRank;
            int aggionid = all_procInfo[tmp_ranklist[i]].ionID;
            int foundrank = 0;
            int summaryranklistbridgeindex = 0;
            int j;
            for (j = 0; (j < numbridges && !foundrank); j++) {
                if (bridgelist[j] == aggbridgerank) {
                    foundrank = 1;
                    summaryranklistbridgeindex = j;
                } else
                    summaryranklistbridgeindex++;
            }
            if (!foundrank) {
                bridgelist[summaryranklistbridgeindex] = aggbridgerank;
                ionlist[summaryranklistbridgeindex] = aggionid;

                if (summarybridgeminionaggrank[summaryranklistbridgeindex] == -1)
                    summarybridgeminionaggrank[summaryranklistbridgeindex] = aggbridgerank;
                else if (summarybridgeminionaggrank[summaryranklistbridgeindex] > aggbridgerank)
                    summarybridgeminionaggrank[summaryranklistbridgeindex] = aggbridgerank;
                numbridges++;
            }

            bridgelistnum[summaryranklistbridgeindex]++;
        }

        /* at this point summarybridgeminionaggrank has the agg rank of the bridge for entries,
         * need to make each entry the minimum bridge rank for the entire ion. */
        for (i = 0; i < numbridges; i++) {
            int aggIonId = ionlist[i];
            int j;
            for (j = 0; j < numbridges; j++) {
                if (ionlist[j] == aggIonId) {
                    if (summarybridgeminionaggrank[j] < summarybridgeminionaggrank[i])
                        summarybridgeminionaggrank[i] = summarybridgeminionaggrank[j];
                }
            }
        }

        // resort by io node minimum bridge rank
        int x;
        for (x = 0; x < numbridges; x++) {
            for (i = 0; i < (numbridges - 1); i++) {
                if (summarybridgeminionaggrank[i] > summarybridgeminionaggrank[i + 1]) {
                    int tmpminionaggrank = summarybridgeminionaggrank[i];
                    summarybridgeminionaggrank[i] = summarybridgeminionaggrank[i + 1];
                    summarybridgeminionaggrank[i + 1] = tmpminionaggrank;
                    int tmpionid = ionlist[i];
                    ionlist[i] = ionlist[i + 1];
                    ionlist[i + 1] = tmpionid;
                    int tmpbridgerank = bridgelist[i];
                    bridgelist[i] = bridgelist[i + 1];
                    bridgelist[i + 1] = tmpbridgerank;
                    int tmpbridgeranknum = bridgelistnum[i];
                    bridgelistnum[i] = bridgelistnum[i + 1];
                    bridgelistnum[i + 1] = tmpbridgeranknum;
                }
            }
        }

        // for each io node make sure bridgelist is in rank order
        int startSortIndex = -1;
        int endSortIndex = -1;
        int currentBridgeIndex = 0;

        while (currentBridgeIndex < numbridges) {
            int currentIonId = ionlist[currentBridgeIndex];
            startSortIndex = currentBridgeIndex;
            while (ionlist[currentBridgeIndex] == currentIonId)
                currentBridgeIndex++;
            endSortIndex = currentBridgeIndex - 1;
            for (x = startSortIndex; x <= endSortIndex; x++) {
                for (i = startSortIndex; i < endSortIndex; i++) {
                    if (bridgelist[i] > bridgelist[i + 1]) {
                        int tmpbridgerank = bridgelist[i];
                        bridgelist[i] = bridgelist[i + 1];
                        bridgelist[i + 1] = tmpbridgerank;
                        int tmpbridgeranknum = bridgelistnum[i];
                        bridgelistnum[i] = bridgelistnum[i + 1];
                        bridgelistnum[i + 1] = tmpbridgeranknum;
                    }
                }
            }
        }


        /* populate interleavedbridgeranklist - essentially the agg rank list
         * is now sorted by the ion minimum bridge rank and bridge node */
        int currentrankoffset = 0;
        for (i = 0; i < numbridges; i++) {
            int *thisBridgeAggList = (int *) ADIOI_Malloc(naggs * sizeof(int));
            int numAggsForThisBridge = 0;

            int k;
            for (k = 0; k < naggs; k++) {
                int aggbridgerank = all_procInfo[tmp_ranklist[k]].bridgeRank;
                if (aggbridgerank == bridgelist[i]) {
                    thisBridgeAggList[numAggsForThisBridge] = tmp_ranklist[k];
                    numAggsForThisBridge++;
                }
            }

            // sort thisBridgeAggList
            for (x = 0; x < numAggsForThisBridge; x++) {
                int n;
                for (n = 0; n < (numAggsForThisBridge - 1); n++) {
                    if (thisBridgeAggList[n] > thisBridgeAggList[n + 1]) {
                        int tmpthisBridgeAggList = thisBridgeAggList[n];
                        thisBridgeAggList[n] = thisBridgeAggList[n + 1];
                        thisBridgeAggList[n + 1] = tmpthisBridgeAggList;
                    }
                }
            }
            int n;
            for (n = 0; n < numAggsForThisBridge; n++) {
                interleavedbridgeranklist[currentrankoffset] = thisBridgeAggList[n];
                currentrankoffset++;
            }
            ADIOI_Free(thisBridgeAggList);
        }

#ifdef balancecontigtrace
        fprintf(stderr, "Interleaved aggregator list:\n");
        for (i = 0; i < naggs; i++) {
            fprintf(stderr, "Agg: %d Agg rank: %d with bridge rank %d and ion ID %d\n", i,
                    interleavedbridgeranklist[i],
                    all_procInfo[interleavedbridgeranklist[i]].bridgeRank,
                    all_procInfo[interleavedbridgeranklist[i]].ionID);
        }
        fprintf(stderr, "Bridges list:\n");
        for (i = 0; i < numbridges; i++) {
            fprintf(stderr, "bridge %d ion min rank %d rank %d number of aggs %d ion id %d\n", i,
                    summarybridgeminionaggrank[i], bridgelist[i], bridgelistnum[i], ionlist[i]);
        }

#endif
        /* copy the ranklist of IO aggregators to fd->hints */
        if (fd->hints->ranklist != NULL)
            ADIOI_Free(fd->hints->ranklist);
        if (fd->hints->fs_hints.bg.bridgelist != NULL)
            ADIOI_Free(fd->hints->fs_hints.bg.bridgelist);
        if (fd->hints->fs_hints.bg.bridgelistnum != NULL)
            ADIOI_Free(fd->hints->fs_hints.bg.bridgelistnum);

        fd->hints->cb_nodes = naggs;
        fd->hints->fs_hints.bg.numbridges = numbridges;
        fd->hints->ranklist = (int *) ADIOI_Malloc(naggs * sizeof(int));
        memcpy(fd->hints->ranklist, interleavedbridgeranklist, naggs * sizeof(int));

        fd->hints->fs_hints.bg.bridgelist = (int *) ADIOI_Malloc(naggs * sizeof(int));
        memcpy(fd->hints->fs_hints.bg.bridgelist, bridgelist, naggs * sizeof(int));

        fd->hints->fs_hints.bg.bridgelistnum = (int *) ADIOI_Malloc(naggs * sizeof(int));
        memcpy(fd->hints->fs_hints.bg.bridgelistnum, bridgelistnum, naggs * sizeof(int));

        ADIOI_Free(summarybridgeminionaggrank);
        ADIOI_Free(tmp_ranklist);
        ADIOI_Free(bridgelistnum);
        ADIOI_Free(bridgelist);
        ADIOI_Free(interleavedbridgeranklist);
        ADIOI_Free(ionlist);

    } else {
        /* classic topology-agnostic copy of the ranklist of IO aggregators to
         * fd->hints */
        if (fd->hints->ranklist != NULL)
            ADIOI_Free(fd->hints->ranklist);

        fd->hints->cb_nodes = naggs;
        fd->hints->ranklist = (int *) ADIOI_Malloc(naggs * sizeof(int));
        memcpy(fd->hints->ranklist, tmp_ranklist, naggs * sizeof(int));

        ADIOI_Free(tmp_ranklist);
    }
    TRACE_ERR("Leaving ADIOI_BG_compute_agg_ranklist_serial\n");
    return;
}
